package com.gsy.wordcount;

import com.gsy.gsy_common_util.fileUtil.DirectoryUtils;
import com.gsy.gsy_common_util.fileUtil.FileUtils;

import java.io.File;
import java.io.IOException;
import java.util.*;

public class CleanDIr {
    public static void main(String[] args) {
        String basedir = "D:\\test\\工科英语\\工科英语\\ITM";
        List<File> files = DirectoryUtils.listAllFiles(new File(basedir), "txt");
        HashSet<String> hashSet = new HashSet<>();
        try {
            String s = FileUtils.fileToString("D:\\test\\工科英语\\stop.txt");
            String[] stop = s.split("\\s");
            for (String s1 : stop) {
                hashSet.add(s1.toLowerCase());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        HashMap<String, Integer> hashMap = new HashMap<>();
        for (File file : files) {
            try {
                String text = FileUtils.fileToString(file);
//            System.out.println(text);
                String a = text.replaceAll("[^a-zA-Z]", " ").replaceAll("\\s+", " ");
                String arr[] = a.split("\\s");

                for (String s : arr) {
                    if (s.matches("[A-Z][a-z]+")) {
                        s = s.toLowerCase();
                    }
                    if (s.length() < 4) {
                        continue;
                    }
                    if (!hashSet.contains(s.toLowerCase())) {
                        hashMap.put(s, hashMap.getOrDefault(s, 0) + 1);
                    }
                }

            } catch (IOException e) {
                e.printStackTrace();
            }


        }
        ArrayList<Node> arrayList = new ArrayList<>();
        for (Map.Entry<String, Integer> stringIntegerEntry : hashMap.entrySet()) {
            arrayList.add(new Node(stringIntegerEntry.getKey(), stringIntegerEntry.getValue()));
        }
        Collections.sort(arrayList);
        StringBuffer stringBuffer = new StringBuffer();
        for (Node node : arrayList) {
//                System.out.println(node);
            stringBuffer.append(node.toString() + "\n");
        }
        try {
            FileUtils.stringToFile(stringBuffer.toString(), basedir,  "ana.csv", "utf-8");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}
